using System;

using Jeffrey.XFramework.XCrawl;

namespace Jeffrey.XFramework
{
    public abstract class Extractor : Processor
    {
        /* 
         * Chinese characters:  \u4E00-\u9FA5
         * Chinese punctuation: \uFE30-\uFFA0
         * 
         * Notice: all of the patterns should use 'RegexOptions.IgnoreCase' flag!
         */

        /*
         * Notice: For Http URL syntax is
         *          
         * URI            = ( absoluteURI | relativeURI ) [ "#" fragment ]

         * absoluteURI    = scheme ":" *( uchar|reserved ) = scheme":"*(unreserved|"%"HEXHEX|";" | "/" | "?" | ":" | "@" | "&" | "=" | "+")

         * relativeURI    = net_path|abs_path|rel_path

         * net_path       = "//" net_loc[ abs_path ]
         * abs_path       = "/" rel_path
         * rel_path       = [ path ][ ";" params ][ "?" query ]

         * path           = fsegment *( "/" segment )
         * fsegment       = 1*pchar
         * segment        = *pchar

         * params         = param *( ";" param )
         * param          = *( pchar | "/" )

         * scheme         = 1*( ALPHA | DIGIT | "+" | "-" | "." )
         * net_loc        = *( pchar | ";" | "?" )

         * query          = *( uchar | reserved )
         * fragment       = *( uchar | reserved )

         * pchar          = uchar | ":" | "@" | "&" | "=" | "+"
         * uchar          = unreserved | escape
         * unreserved     = ALPHA | DIGIT | safe | extra | national

         * escape         = "%" HEX HEX
         * reserved       = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+"
         * extra          = "!" | "*" | "'" | "(" | ")" | ","
         * safe           = "$" | "-" | "_" | "."
         * unsafe         = CTL | SP | <"> | "#" | "%" | "<" | ">"
         * national       = <any OCTET excluding ALPHA, DIGIT,
                            reserved, extra, safe, and unsafe>
         * 
         */
        protected static readonly String URI_EXCLUDE_CHARS = @"\<> '""{}|,^\[\]\u3000\u4E00-\u9FA5\uFE30-\uFFA0";

        //
        protected static readonly String HTTP_URI_VALIDATE_PATTERN = @"^http://[^" + URI_EXCLUDE_CHARS + @"]+$";

        //the main pattern of http:// schema...
        protected static readonly String HTTP_URI_EXTRACT_PATTERN  = @"\bhttp://[^" + URI_EXCLUDE_CHARS + @"]+\b";

        //the pattern to extrac urls from <a href=...></a>
        protected static readonly String HTTP_URI_EX_EXTRACT_PATTERN = @"<a +[^>]*href *=[ ""']*(?<uri>[^" + URI_EXCLUDE_CHARS + "]+)[^>]*>";

        /*
         * the html extentions...
         * for some urls wiht query part "?", need absolutely-path to match this pattern.
         */
        protected static readonly String HTML_EXTENSIONS = @".+\.(s?html?|aspx?|jsp|php)$";

        public static String UriValidatePattern { get { return HTTP_URI_VALIDATE_PATTERN; } }
        public static String HtmlFilePattern { get { return HTML_EXTENSIONS; } }

        public Extractor(String name, String description, String classname, String path)
            : base(name, description, classname, path, null) { }
        public Extractor(String name, String description, String classname, String path, Filter filter)
            : base(name, description, classname, path, filter) { }

        protected abstract void Extract(CrawlUri uri);

        protected override sealed void innerProcess(CrawlUri uri)
        {
            try
            {
                Extract(uri);
            }
            catch (NullReferenceException)
            {
                uri.m_Status = CrawlUri.StatusCode.E_Extract;
            }
            catch (StackOverflowException)
            {
                uri.m_Status = CrawlUri.StatusCode.E_Extract;
            }
        }
        protected override void innerRejectProcess(CrawlUri curi)
        {
            /*
             * by default do nothing...
             */
        }
    }
}
